{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path as ops\n",
    "import numpy as np\n",
    "import torch\n",
    "import cv2\n",
    "import time\n",
    "from dataset.dataset_utils import TUSIMPLE\n",
    "from Lanenet.model2 import Lanenet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_set length 3082\n",
      "valid_set length 181\n",
      "test_set length 363\n"
     ]
    }
   ],
   "source": [
    "# root = '/Users/smiffy/Documents/GitHub/TUSIMPLE/Data_Tusimple_PyTorch/training'\n",
    "root = 'TUSIMPLE/txt_for_local'\n",
    "train_set = TUSIMPLE(root=root, flag='train')\n",
    "valid_set = TUSIMPLE(root=root, flag='valid')\n",
    "test_set = TUSIMPLE(root=root, flag='test')\n",
    "\n",
    "print('train_set length {}'.format(len(train_set)))\n",
    "print('valid_set length {}'.format(len(valid_set)))\n",
    "print('test_set length {}'.format(len(test_set)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image type <class 'torch.Tensor'>\n",
      "image size torch.Size([3, 256, 512]) \n",
      "\n",
      "gt binary image type <class 'torch.Tensor'>\n",
      "gt binary image size torch.Size([256, 512])\n",
      "items in gt binary image tensor([0, 1]) \n",
      "\n",
      "gt instance type <class 'torch.Tensor'>\n",
      "gt instance size torch.Size([256, 512])\n",
      "items in gt instance tensor([  0,  20,  70, 120, 170]) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gt, bgt, igt = train_set[0]\n",
    "print('image type {}'.format(type(gt)))\n",
    "print('image size {} \\n'.format(gt.size()))\n",
    "\n",
    "print('gt binary image type {}'.format(type(bgt)))\n",
    "print('gt binary image size {}'.format(bgt.size()))\n",
    "print('items in gt binary image {} \\n'.format(torch.unique(bgt)))\n",
    "\n",
    "print('gt instance type {}'.format(type(igt)))\n",
    "print('gt instance size {}'.format(igt.size()))\n",
    "print('items in gt instance {} \\n'.format(torch.unique(igt)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader_train = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "data_loader_valid = torch.utils.data.DataLoader(valid_set, batch_size=1, shuffle=True, num_workers=0)\n",
    "data_loader_test = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model and optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 5e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "LaneNet_model = Lanenet(2, 4)\n",
    "LaneNet_model.to(device)\n",
    "\n",
    "params = [p for p in LaneNet_model.parameters() if p.requires_grad]\n",
    "optimizer = torch.optim.Adam(params, lr=learning_rate, weight_decay=0.0002)\n",
    "\n",
    "lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Lanenet.cluster_loss3 import cluster_loss\n",
    "criterion = cluster_loss()\n",
    "# criterion = torch.nn.CrossEntropyLoss(weight=torch.tensor([ 1.4393, 27.7296]).cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[0] iter[0] loss: [0.7717495560646057, 9.4901704788208] \n",
      "epoch[0] iter[20] loss: [0.6380378603935242, 7.957880020141602] \n",
      "epoch[0] iter[40] loss: [0.5467560291290283, 5.812410831451416] \n",
      "epoch[0] iter[60] loss: [0.47288209199905396, 4.490697860717773] \n",
      "epoch[0] iter[80] loss: [0.42886632680892944, 3.3589000701904297] \n",
      "epoch[0] iter[100] loss: [0.3896348476409912, 2.362640142440796] \n",
      "epoch[0] iter[120] loss: [0.33741384744644165, 1.4766401052474976] \n",
      "epoch[0] iter[140] loss: [0.3143121898174286, 1.5628695487976074] \n",
      "epoch[0] iter[160] loss: [0.26901382207870483, 1.4824877977371216] \n",
      "epoch[0] iter[180] loss: [0.3082256317138672, 1.6616493463516235] \n",
      "epoch[0] iter[200] loss: [0.2668899893760681, 1.422430157661438] \n",
      "epoch[0] iter[220] loss: [0.28200364112854004, 1.6095092296600342] \n",
      "epoch[0] iter[240] loss: [0.2237468659877777, 1.3117060661315918] \n",
      "epoch[0] iter[260] loss: [0.2257433831691742, 1.0865285396575928] \n",
      "epoch[0] iter[280] loss: [0.2212977558374405, 0.9034909605979919] \n",
      "epoch[0] iter[300] loss: [0.21762850880622864, 1.2305874824523926] \n",
      "epoch[0] iter[320] loss: [0.17390669882297516, 0.9292185306549072] \n",
      "epoch[0] iter[340] loss: [0.18391233682632446, 0.8957005739212036] \n",
      "epoch[0] iter[360] loss: [0.21209120750427246, 0.9317983388900757] \n",
      "epoch[0] iter[380] loss: [0.20464837551116943, 0.8780602812767029] \n",
      "Finish epoch[0], time elapsed[622.5130681991577]\n",
      "epoch[1] iter[0] loss: [0.21405069530010223, 0.9175050854682922] \n",
      "epoch[1] iter[20] loss: [0.1854901909828186, 0.5410380363464355] \n",
      "epoch[1] iter[40] loss: [0.19675953686237335, 0.6333016157150269] \n",
      "epoch[1] iter[60] loss: [0.20916800200939178, 0.5470629930496216] \n",
      "epoch[1] iter[80] loss: [0.14253082871437073, 0.47784706950187683] \n",
      "epoch[1] iter[100] loss: [0.1880076378583908, 0.859294056892395] \n",
      "epoch[1] iter[120] loss: [0.18693090975284576, 0.5444453954696655] \n",
      "epoch[1] iter[140] loss: [0.20232857763767242, 0.4389948546886444] \n",
      "epoch[1] iter[160] loss: [0.14407978951931, 0.3194066882133484] \n",
      "epoch[1] iter[180] loss: [0.1569238007068634, 0.32706940174102783] \n",
      "epoch[1] iter[200] loss: [0.1836182177066803, 0.3839263916015625] \n",
      "epoch[1] iter[220] loss: [0.15119406580924988, 0.2877209782600403] \n",
      "epoch[1] iter[240] loss: [0.13407300412654877, 0.35436898469924927] \n",
      "epoch[1] iter[260] loss: [0.13825616240501404, 0.23273064196109772] \n",
      "epoch[1] iter[280] loss: [0.1586926132440567, 0.29186394810676575] \n",
      "epoch[1] iter[300] loss: [0.16106581687927246, 0.2885951101779938] \n",
      "epoch[1] iter[320] loss: [0.1665404886007309, 0.373661607503891] \n",
      "epoch[1] iter[340] loss: [0.14221614599227905, 0.22131164371967316] \n",
      "epoch[1] iter[360] loss: [0.14617419242858887, 0.21848216652870178] \n",
      "epoch[1] iter[380] loss: [0.1480700969696045, 0.29213130474090576] \n",
      "Finish epoch[1], time elapsed[282.58361887931824]\n",
      "epoch[2] iter[0] loss: [0.15794888138771057, 0.26162993907928467] \n",
      "epoch[2] iter[20] loss: [0.12232654541730881, 0.21671940386295319] \n",
      "epoch[2] iter[40] loss: [0.15599606931209564, 0.31900736689567566] \n",
      "epoch[2] iter[60] loss: [0.17949019372463226, 0.3763277232646942] \n",
      "epoch[2] iter[80] loss: [0.12183533608913422, 0.187861829996109] \n",
      "epoch[2] iter[100] loss: [0.1403336226940155, 0.24115687608718872] \n",
      "epoch[2] iter[120] loss: [0.1792178601026535, 0.24888740479946136] \n",
      "epoch[2] iter[140] loss: [0.15507932007312775, 0.3568210303783417] \n",
      "epoch[2] iter[160] loss: [0.1455875039100647, 0.2176668494939804] \n",
      "epoch[2] iter[180] loss: [0.13826622068881989, 0.391392320394516] \n",
      "epoch[2] iter[200] loss: [0.1680276244878769, 0.391460120677948] \n",
      "epoch[2] iter[220] loss: [0.1599358469247818, 0.2068808674812317] \n",
      "epoch[2] iter[240] loss: [0.13086891174316406, 0.3806004524230957] \n",
      "epoch[2] iter[260] loss: [0.15220822393894196, 0.2334069162607193] \n",
      "epoch[2] iter[280] loss: [0.17178292572498322, 0.4304737150669098] \n",
      "epoch[2] iter[300] loss: [0.13527782261371613, 0.20661529898643494] \n",
      "epoch[2] iter[320] loss: [0.1401335597038269, 0.22756348550319672] \n",
      "epoch[2] iter[340] loss: [0.12981849908828735, 0.25216567516326904] \n",
      "epoch[2] iter[360] loss: [0.15891972184181213, 0.24995964765548706] \n",
      "epoch[2] iter[380] loss: [0.13119789958000183, 0.2887823283672333] \n",
      "Finish epoch[2], time elapsed[282.0691542625427]\n",
      "epoch[3] iter[0] loss: [0.15216024219989777, 0.30461549758911133] \n",
      "epoch[3] iter[20] loss: [0.14888925850391388, 0.30868107080459595] \n",
      "epoch[3] iter[40] loss: [0.13922907412052155, 0.19577337801456451] \n",
      "epoch[3] iter[60] loss: [0.13949711620807648, 0.18696199357509613] \n",
      "epoch[3] iter[80] loss: [0.11960664391517639, 0.14904020726680756] \n",
      "epoch[3] iter[100] loss: [0.1290111392736435, 0.19242790341377258] \n",
      "epoch[3] iter[120] loss: [0.13358698785305023, 0.252107173204422] \n",
      "epoch[3] iter[140] loss: [0.14327360689640045, 0.2622400224208832] \n",
      "epoch[3] iter[160] loss: [0.16123400628566742, 0.372200608253479] \n",
      "epoch[3] iter[180] loss: [0.14568831026554108, 0.213498055934906] \n",
      "epoch[3] iter[200] loss: [0.12238570302724838, 0.188766747713089] \n",
      "epoch[3] iter[220] loss: [0.1372399479150772, 0.16114211082458496] \n",
      "epoch[3] iter[240] loss: [0.14053629338741302, 0.19885683059692383] \n",
      "epoch[3] iter[260] loss: [0.1684773713350296, 0.29476770758628845] \n",
      "epoch[3] iter[280] loss: [0.18165966868400574, 0.4461194574832916] \n",
      "epoch[3] iter[300] loss: [0.12251107394695282, 0.20018264651298523] \n",
      "epoch[3] iter[320] loss: [0.10910993814468384, 0.11136740446090698] \n",
      "epoch[3] iter[340] loss: [0.15408624708652496, 0.20295017957687378] \n",
      "epoch[3] iter[360] loss: [0.11401157826185226, 0.15926018357276917] \n",
      "epoch[3] iter[380] loss: [0.12945188581943512, 0.14335539937019348] \n",
      "Finish epoch[3], time elapsed[282.91488003730774]\n",
      "epoch[4] iter[0] loss: [0.11023371666669846, 0.18421927094459534] \n",
      "epoch[4] iter[20] loss: [0.12342371046543121, 0.194215789437294] \n",
      "epoch[4] iter[40] loss: [0.1228037178516388, 0.1420826017856598] \n",
      "epoch[4] iter[60] loss: [0.11534672230482101, 0.19259177148342133] \n",
      "epoch[4] iter[80] loss: [0.12648233771324158, 0.25635650753974915] \n",
      "epoch[4] iter[100] loss: [0.1360519528388977, 0.21298183500766754] \n",
      "epoch[4] iter[120] loss: [0.11060746759176254, 0.19203846156597137] \n",
      "epoch[4] iter[140] loss: [0.14984409511089325, 0.29723256826400757] \n",
      "epoch[4] iter[160] loss: [0.13388045132160187, 0.30064278841018677] \n",
      "epoch[4] iter[180] loss: [0.12365296483039856, 0.1594206988811493] \n",
      "epoch[4] iter[200] loss: [0.12574779987335205, 0.1433251053094864] \n",
      "epoch[4] iter[220] loss: [0.12858055531978607, 0.26269790530204773] \n",
      "epoch[4] iter[240] loss: [0.1326262205839157, 0.20213402807712555] \n",
      "epoch[4] iter[260] loss: [0.12759961187839508, 0.15399184823036194] \n",
      "epoch[4] iter[280] loss: [0.13347512483596802, 0.319672554731369] \n",
      "epoch[4] iter[300] loss: [0.11983705312013626, 0.12955635786056519] \n",
      "epoch[4] iter[320] loss: [0.11156618595123291, 0.15610720217227936] \n",
      "epoch[4] iter[340] loss: [0.12090963125228882, 0.1953449845314026] \n",
      "epoch[4] iter[360] loss: [0.11689160019159317, 0.1715812236070633] \n",
      "epoch[4] iter[380] loss: [0.12123511731624603, 0.23934873938560486] \n",
      "Finish epoch[4], time elapsed[282.12705302238464]\n",
      "epoch[5] iter[0] loss: [0.10685648024082184, 0.11346926540136337] \n",
      "epoch[5] iter[20] loss: [0.1331283152103424, 0.18454593420028687] \n",
      "epoch[5] iter[40] loss: [0.1334894299507141, 0.14108321070671082] \n",
      "epoch[5] iter[60] loss: [0.12430886924266815, 0.2781546711921692] \n",
      "epoch[5] iter[80] loss: [0.11765719950199127, 0.1802087426185608] \n",
      "epoch[5] iter[100] loss: [0.10288317501544952, 0.12102311849594116] \n",
      "epoch[5] iter[120] loss: [0.12891468405723572, 0.21938198804855347] \n",
      "epoch[5] iter[140] loss: [0.12366611510515213, 0.16603736579418182] \n",
      "epoch[5] iter[160] loss: [0.10905742645263672, 0.1866701990365982] \n",
      "epoch[5] iter[180] loss: [0.11558876186609268, 0.24144530296325684] \n",
      "epoch[5] iter[200] loss: [0.14427168667316437, 0.26961445808410645] \n",
      "epoch[5] iter[220] loss: [0.1104586198925972, 0.16224759817123413] \n",
      "epoch[5] iter[240] loss: [0.1250280737876892, 0.1857253909111023] \n",
      "epoch[5] iter[260] loss: [0.1194785088300705, 0.13031648099422455] \n",
      "epoch[5] iter[280] loss: [0.12701644003391266, 0.16599097847938538] \n",
      "epoch[5] iter[300] loss: [0.1066465750336647, 0.12947627902030945] \n",
      "epoch[5] iter[320] loss: [0.14305371046066284, 0.21652987599372864] \n",
      "epoch[5] iter[340] loss: [0.11598512530326843, 0.1718219369649887] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[5] iter[360] loss: [0.14442257583141327, 0.16854755580425262] \n",
      "epoch[5] iter[380] loss: [0.12781338393688202, 0.18122552335262299] \n",
      "Finish epoch[5], time elapsed[282.64349365234375]\n",
      "epoch[6] iter[0] loss: [0.1269368678331375, 0.18246546387672424] \n",
      "epoch[6] iter[20] loss: [0.13363920152187347, 0.2041665017604828] \n",
      "epoch[6] iter[40] loss: [0.10343474894762039, 0.14970853924751282] \n",
      "epoch[6] iter[60] loss: [0.14442235231399536, 0.16686135530471802] \n",
      "epoch[6] iter[80] loss: [0.11887942254543304, 0.1607373207807541] \n",
      "epoch[6] iter[100] loss: [0.10740653425455093, 0.14503328502178192] \n",
      "epoch[6] iter[120] loss: [0.12073865532875061, 0.15439678728580475] \n",
      "epoch[6] iter[140] loss: [0.09412746876478195, 0.1474987268447876] \n",
      "epoch[6] iter[160] loss: [0.12857629358768463, 0.31585949659347534] \n",
      "epoch[6] iter[180] loss: [0.10584688186645508, 0.11505185067653656] \n",
      "epoch[6] iter[200] loss: [0.12073928117752075, 0.14640964567661285] \n",
      "epoch[6] iter[220] loss: [0.11235077679157257, 0.11871308088302612] \n",
      "epoch[6] iter[240] loss: [0.12723149359226227, 0.1812680959701538] \n",
      "epoch[6] iter[260] loss: [0.16806018352508545, 0.26849254965782166] \n",
      "epoch[6] iter[280] loss: [0.12931963801383972, 0.15890942513942719] \n",
      "epoch[6] iter[300] loss: [0.12797005474567413, 0.1437491774559021] \n",
      "epoch[6] iter[320] loss: [0.11175081878900528, 0.15886399149894714] \n",
      "epoch[6] iter[340] loss: [0.14357538521289825, 0.14263267815113068] \n",
      "epoch[6] iter[360] loss: [0.12579183280467987, 0.16127032041549683] \n",
      "epoch[6] iter[380] loss: [0.13602158427238464, 0.1628655642271042] \n",
      "Finish epoch[6], time elapsed[283.1162164211273]\n",
      "epoch[7] iter[0] loss: [0.11476535350084305, 0.11955267935991287] \n",
      "epoch[7] iter[20] loss: [0.11085765808820724, 0.1373417228460312] \n",
      "epoch[7] iter[40] loss: [0.10126539319753647, 0.12315153330564499] \n",
      "epoch[7] iter[60] loss: [0.10707245022058487, 0.11867214739322662] \n",
      "epoch[7] iter[80] loss: [0.13377131521701813, 0.21614083647727966] \n",
      "epoch[7] iter[100] loss: [0.12535539269447327, 0.1761559098958969] \n",
      "epoch[7] iter[120] loss: [0.13965974748134613, 0.1560799926519394] \n",
      "epoch[7] iter[140] loss: [0.11176851391792297, 0.16600921750068665] \n",
      "epoch[7] iter[160] loss: [0.1027073860168457, 0.25878649950027466] \n",
      "epoch[7] iter[180] loss: [0.11636269837617874, 0.1393427699804306] \n",
      "epoch[7] iter[200] loss: [0.11008937656879425, 0.1819882094860077] \n",
      "epoch[7] iter[220] loss: [0.10524885356426239, 0.09829563647508621] \n",
      "epoch[7] iter[240] loss: [0.10997945815324783, 0.11533063650131226] \n",
      "epoch[7] iter[260] loss: [0.12013686448335648, 0.1211242824792862] \n",
      "epoch[7] iter[280] loss: [0.11062655597925186, 0.1572578251361847] \n",
      "epoch[7] iter[300] loss: [0.12586313486099243, 0.12989512085914612] \n",
      "epoch[7] iter[320] loss: [0.1365002989768982, 0.161939337849617] \n",
      "epoch[7] iter[340] loss: [0.11982271820306778, 0.13754567503929138] \n",
      "epoch[7] iter[360] loss: [0.10939645767211914, 0.15731702744960785] \n",
      "epoch[7] iter[380] loss: [0.1098235547542572, 0.14222712814807892] \n",
      "Finish epoch[7], time elapsed[281.85643315315247]\n",
      "epoch[8] iter[0] loss: [0.10939908772706985, 0.13903799653053284] \n",
      "epoch[8] iter[20] loss: [0.11840653419494629, 0.15166975557804108] \n",
      "epoch[8] iter[40] loss: [0.13161888718605042, 0.30187132954597473] \n",
      "epoch[8] iter[60] loss: [0.12872925400733948, 0.2013459950685501] \n",
      "epoch[8] iter[80] loss: [0.11578384041786194, 0.12918399274349213] \n",
      "epoch[8] iter[100] loss: [0.1370428204536438, 0.23539328575134277] \n",
      "epoch[8] iter[120] loss: [0.12324680387973785, 0.160013347864151] \n",
      "epoch[8] iter[140] loss: [0.11251413077116013, 0.14477524161338806] \n",
      "epoch[8] iter[160] loss: [0.10973860323429108, 0.1170361116528511] \n",
      "epoch[8] iter[180] loss: [0.11537045985460281, 0.12926822900772095] \n",
      "epoch[8] iter[200] loss: [0.12598374485969543, 0.13540415465831757] \n",
      "epoch[8] iter[220] loss: [0.12433271110057831, 0.1432647407054901] \n",
      "epoch[8] iter[240] loss: [0.1161862388253212, 0.15895625948905945] \n",
      "epoch[8] iter[260] loss: [0.10434722155332565, 0.1642237901687622] \n",
      "epoch[8] iter[280] loss: [0.13213379681110382, 0.18357159197330475] \n",
      "epoch[8] iter[300] loss: [0.11067397892475128, 0.11004288494586945] \n",
      "epoch[8] iter[320] loss: [0.13144688308238983, 0.15687832236289978] \n",
      "epoch[8] iter[340] loss: [0.11877693235874176, 0.14153404533863068] \n",
      "epoch[8] iter[360] loss: [0.1101115271449089, 0.1238289475440979] \n",
      "epoch[8] iter[380] loss: [0.15133629739284515, 0.18237197399139404] \n",
      "Finish epoch[8], time elapsed[281.26211404800415]\n",
      "epoch[9] iter[0] loss: [0.12812969088554382, 0.16300661861896515] \n",
      "epoch[9] iter[20] loss: [0.1183885857462883, 0.13481317460536957] \n",
      "epoch[9] iter[40] loss: [0.09119250625371933, 0.08993849158287048] \n",
      "epoch[9] iter[60] loss: [0.11781548708677292, 0.17108166217803955] \n",
      "epoch[9] iter[80] loss: [0.11621323227882385, 0.22275923192501068] \n",
      "epoch[9] iter[100] loss: [0.0961148589849472, 0.09666897356510162] \n",
      "epoch[9] iter[120] loss: [0.09709297120571136, 0.11505807936191559] \n",
      "epoch[9] iter[140] loss: [0.11013951152563095, 0.10942672938108444] \n",
      "epoch[9] iter[160] loss: [0.10029027611017227, 0.13147948682308197] \n",
      "epoch[9] iter[180] loss: [0.10676723718643188, 0.12961538136005402] \n",
      "epoch[9] iter[200] loss: [0.1120893806219101, 0.11598417907953262] \n",
      "epoch[9] iter[220] loss: [0.10741331428289413, 0.1417798399925232] \n",
      "epoch[9] iter[240] loss: [0.10825170576572418, 0.11217246204614639] \n",
      "epoch[9] iter[260] loss: [0.09873384982347488, 0.10318737477064133] \n",
      "epoch[9] iter[280] loss: [0.09705059975385666, 0.17508810758590698] \n",
      "epoch[9] iter[300] loss: [0.13042184710502625, 0.16473519802093506] \n",
      "epoch[9] iter[320] loss: [0.09972891211509705, 0.10739388316869736] \n",
      "epoch[9] iter[340] loss: [0.10229943692684174, 0.09552336484193802] \n",
      "epoch[9] iter[360] loss: [0.1318269670009613, 0.16283021867275238] \n",
      "epoch[9] iter[380] loss: [0.11653552204370499, 0.13053493201732635] \n",
      "Finish epoch[9], time elapsed[282.1175901889801]\n",
      "epoch[10] iter[0] loss: [0.10398031026124954, 0.1319793313741684] \n",
      "epoch[10] iter[20] loss: [0.10505741089582443, 0.11082065105438232] \n",
      "epoch[10] iter[40] loss: [0.13187964260578156, 0.18006449937820435] \n",
      "epoch[10] iter[60] loss: [0.10491669923067093, 0.11494027078151703] \n",
      "epoch[10] iter[80] loss: [0.11508084088563919, 0.13055337965488434] \n",
      "epoch[10] iter[100] loss: [0.09848551452159882, 0.12305836379528046] \n",
      "epoch[10] iter[120] loss: [0.10859168320894241, 0.13721558451652527] \n",
      "epoch[10] iter[140] loss: [0.15946117043495178, 0.27271097898483276] \n",
      "epoch[10] iter[160] loss: [0.11803355067968369, 0.16046854853630066] \n",
      "epoch[10] iter[180] loss: [0.12205977737903595, 0.17403310537338257] \n",
      "epoch[10] iter[200] loss: [0.10900816321372986, 0.11793124675750732] \n",
      "epoch[10] iter[220] loss: [0.10479403287172318, 0.12945178151130676] \n",
      "epoch[10] iter[240] loss: [0.13317863643169403, 0.130711629986763] \n",
      "epoch[10] iter[260] loss: [0.12601043283939362, 0.1682175099849701] \n",
      "epoch[10] iter[280] loss: [0.11936429888010025, 0.14594508707523346] \n",
      "epoch[10] iter[300] loss: [0.1151203066110611, 0.12195828557014465] \n",
      "epoch[10] iter[320] loss: [0.12772966921329498, 0.1913907378911972] \n",
      "epoch[10] iter[340] loss: [0.10112199187278748, 0.15114617347717285] \n",
      "epoch[10] iter[360] loss: [0.09609602391719818, 0.1104864776134491] \n",
      "epoch[10] iter[380] loss: [0.1060899868607521, 0.14838893711566925] \n",
      "Finish epoch[10], time elapsed[281.5580050945282]\n",
      "epoch[11] iter[0] loss: [0.11058303713798523, 0.1095566526055336] \n",
      "epoch[11] iter[20] loss: [0.11383450031280518, 0.13836437463760376] \n",
      "epoch[11] iter[40] loss: [0.11028753221035004, 0.12087763100862503] \n",
      "epoch[11] iter[60] loss: [0.10421140491962433, 0.09039199352264404] \n",
      "epoch[11] iter[80] loss: [0.09795504808425903, 0.10516276210546494] \n",
      "epoch[11] iter[100] loss: [0.09709086269140244, 0.10069351643323898] \n",
      "epoch[11] iter[120] loss: [0.10662005096673965, 0.13157296180725098] \n",
      "epoch[11] iter[140] loss: [0.10991381853818893, 0.08291997760534286] \n",
      "epoch[11] iter[160] loss: [0.10422102361917496, 0.09736195206642151] \n",
      "epoch[11] iter[180] loss: [0.1169615238904953, 0.12654246389865875] \n",
      "epoch[11] iter[200] loss: [0.11145883053541183, 0.1074661910533905] \n",
      "epoch[11] iter[220] loss: [0.10402736067771912, 0.12851206958293915] \n",
      "epoch[11] iter[240] loss: [0.11154496669769287, 0.1273953765630722] \n",
      "epoch[11] iter[260] loss: [0.1112024113535881, 0.10477563738822937] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[11] iter[280] loss: [0.13584966957569122, 0.22066248953342438] \n",
      "epoch[11] iter[300] loss: [0.11660125851631165, 0.11252523213624954] \n",
      "epoch[11] iter[320] loss: [0.10287411510944366, 0.10837321728467941] \n",
      "epoch[11] iter[340] loss: [0.09991713613271713, 0.09056923538446426] \n",
      "epoch[11] iter[360] loss: [0.12162812799215317, 0.19063182175159454] \n",
      "epoch[11] iter[380] loss: [0.11441101878881454, 0.14324216544628143] \n",
      "Finish epoch[11], time elapsed[283.7197358608246]\n",
      "epoch[12] iter[0] loss: [0.10950276255607605, 0.11474771797657013] \n",
      "epoch[12] iter[20] loss: [0.14818546175956726, 0.18601253628730774] \n",
      "epoch[12] iter[40] loss: [0.10348106920719147, 0.10870727896690369] \n",
      "epoch[12] iter[60] loss: [0.09766995161771774, 0.09714238345623016] \n",
      "epoch[12] iter[80] loss: [0.10364972800016403, 0.10951593518257141] \n",
      "epoch[12] iter[100] loss: [0.09330463409423828, 0.11431411653757095] \n",
      "epoch[12] iter[120] loss: [0.11446413397789001, 0.10781621187925339] \n",
      "epoch[12] iter[140] loss: [0.10012227296829224, 0.10647421330213547] \n",
      "epoch[12] iter[160] loss: [0.110038161277771, 0.09073514491319656] \n",
      "epoch[12] iter[180] loss: [0.11886779218912125, 0.15531490743160248] \n",
      "epoch[12] iter[200] loss: [0.1014954000711441, 0.09961975365877151] \n",
      "epoch[12] iter[220] loss: [0.10764482617378235, 0.09249967336654663] \n",
      "epoch[12] iter[240] loss: [0.10869530588388443, 0.1354217529296875] \n",
      "epoch[12] iter[260] loss: [0.12628968060016632, 0.1232929527759552] \n",
      "epoch[12] iter[280] loss: [0.12784305214881897, 0.11907069385051727] \n",
      "epoch[12] iter[300] loss: [0.09163413196802139, 0.10108490288257599] \n",
      "epoch[12] iter[320] loss: [0.09664328396320343, 0.10726146399974823] \n",
      "epoch[12] iter[340] loss: [0.10842984914779663, 0.12578696012496948] \n",
      "epoch[12] iter[360] loss: [0.09873852878808975, 0.0870385393500328] \n",
      "epoch[12] iter[380] loss: [0.12134263664484024, 0.11751769483089447] \n",
      "Finish epoch[12], time elapsed[283.16017508506775]\n",
      "epoch[13] iter[0] loss: [0.13174349069595337, 0.15989816188812256] \n",
      "epoch[13] iter[20] loss: [0.10285413265228271, 0.10023140907287598] \n",
      "epoch[13] iter[40] loss: [0.1056022047996521, 0.09081454575061798] \n",
      "epoch[13] iter[60] loss: [0.11430101096630096, 0.13553160429000854] \n",
      "epoch[13] iter[80] loss: [0.1024879738688469, 0.11245876550674438] \n",
      "epoch[13] iter[100] loss: [0.11414715647697449, 0.13286535441875458] \n",
      "epoch[13] iter[120] loss: [0.10087568312883377, 0.1009797528386116] \n",
      "epoch[13] iter[140] loss: [0.10258748382329941, 0.1027936339378357] \n",
      "epoch[13] iter[160] loss: [0.111429862678051, 0.12729352712631226] \n",
      "epoch[13] iter[180] loss: [0.1135285273194313, 0.11831768602132797] \n",
      "epoch[13] iter[200] loss: [0.11278986185789108, 0.09917406737804413] \n",
      "epoch[13] iter[220] loss: [0.12094398587942123, 0.1326753795146942] \n",
      "epoch[13] iter[240] loss: [0.10356760025024414, 0.10330013185739517] \n",
      "epoch[13] iter[260] loss: [0.11120729893445969, 0.09646271914243698] \n",
      "epoch[13] iter[280] loss: [0.1256917268037796, 0.19518229365348816] \n",
      "epoch[13] iter[300] loss: [0.10920050740242004, 0.11983775347471237] \n",
      "epoch[13] iter[320] loss: [0.11537867039442062, 0.1188318133354187] \n",
      "epoch[13] iter[340] loss: [0.10062264651060104, 0.10240643471479416] \n",
      "epoch[13] iter[360] loss: [0.10644540935754776, 0.11115790903568268] \n",
      "epoch[13] iter[380] loss: [0.09569352865219116, 0.09811762720346451] \n",
      "Finish epoch[13], time elapsed[283.5018095970154]\n",
      "epoch[14] iter[0] loss: [0.11129631847143173, 0.12985721230506897] \n",
      "epoch[14] iter[20] loss: [0.09513470530509949, 0.12056480348110199] \n",
      "epoch[14] iter[40] loss: [0.11532469838857651, 0.13169187307357788] \n",
      "epoch[14] iter[60] loss: [0.10693025588989258, 0.0943262130022049] \n",
      "epoch[14] iter[80] loss: [0.10461049526929855, 0.115022674202919] \n",
      "epoch[14] iter[100] loss: [0.10283268243074417, 0.10868853330612183] \n",
      "epoch[14] iter[120] loss: [0.09198322892189026, 0.09092871844768524] \n",
      "epoch[14] iter[140] loss: [0.09509491920471191, 0.12560313940048218] \n",
      "epoch[14] iter[160] loss: [0.1026652529835701, 0.10752074420452118] \n",
      "epoch[14] iter[180] loss: [0.11248870939016342, 0.10684700310230255] \n",
      "epoch[14] iter[200] loss: [0.1052466407418251, 0.0967225506901741] \n",
      "epoch[14] iter[220] loss: [0.1197420284152031, 0.22514861822128296] \n",
      "epoch[14] iter[240] loss: [0.10394968837499619, 0.10965195298194885] \n",
      "epoch[14] iter[260] loss: [0.08808314800262451, 0.0928221046924591] \n",
      "epoch[14] iter[280] loss: [0.0914771780371666, 0.0978737473487854] \n",
      "epoch[14] iter[300] loss: [0.10150109976530075, 0.09632635116577148] \n",
      "epoch[14] iter[320] loss: [0.09874098747968674, 0.12035723030567169] \n",
      "epoch[14] iter[340] loss: [0.09953083842992783, 0.0821160078048706] \n",
      "epoch[14] iter[360] loss: [0.09485454857349396, 0.10725661367177963] \n",
      "epoch[14] iter[380] loss: [0.13452459871768951, 0.14953511953353882] \n",
      "Finish epoch[14], time elapsed[283.34695172309875]\n",
      "epoch[15] iter[0] loss: [0.12387665361166, 0.15939496457576752] \n",
      "epoch[15] iter[20] loss: [0.11374980211257935, 0.11818626523017883] \n",
      "epoch[15] iter[40] loss: [0.08897832036018372, 0.1006658673286438] \n",
      "epoch[15] iter[60] loss: [0.10374296456575394, 0.10215970873832703] \n",
      "epoch[15] iter[80] loss: [0.09713339805603027, 0.0858646035194397] \n",
      "epoch[15] iter[100] loss: [0.10473904013633728, 0.13016937673091888] \n",
      "epoch[15] iter[120] loss: [0.08843296021223068, 0.07907722890377045] \n",
      "epoch[15] iter[140] loss: [0.09868861734867096, 0.08779917657375336] \n",
      "epoch[15] iter[160] loss: [0.10053224116563797, 0.12433657795190811] \n",
      "epoch[15] iter[180] loss: [0.09534183889627457, 0.09054210782051086] \n",
      "epoch[15] iter[200] loss: [0.11140692979097366, 0.10123318433761597] \n",
      "epoch[15] iter[220] loss: [0.12546247243881226, 0.10585690289735794] \n",
      "epoch[15] iter[240] loss: [0.0904393270611763, 0.09155924618244171] \n",
      "epoch[15] iter[260] loss: [0.12058687210083008, 0.11771631240844727] \n",
      "epoch[15] iter[280] loss: [0.10447150468826294, 0.1209939569234848] \n",
      "epoch[15] iter[300] loss: [0.10910830646753311, 0.12032539397478104] \n",
      "epoch[15] iter[320] loss: [0.10119367390871048, 0.09838440269231796] \n",
      "epoch[15] iter[340] loss: [0.105127714574337, 0.11907771974802017] \n",
      "epoch[15] iter[360] loss: [0.11617488414049149, 0.10060954093933105] \n",
      "epoch[15] iter[380] loss: [0.10533503443002701, 0.10309084504842758] \n",
      "Finish epoch[15], time elapsed[282.50046586990356]\n",
      "epoch[16] iter[0] loss: [0.10759025067090988, 0.11291302740573883] \n",
      "epoch[16] iter[20] loss: [0.1438176929950714, 0.20540815591812134] \n",
      "epoch[16] iter[40] loss: [0.10670098662376404, 0.10270562022924423] \n",
      "epoch[16] iter[60] loss: [0.11288250237703323, 0.15282341837882996] \n",
      "epoch[16] iter[80] loss: [0.11511716991662979, 0.12093569338321686] \n",
      "epoch[16] iter[100] loss: [0.08362670242786407, 0.08893704414367676] \n",
      "epoch[16] iter[120] loss: [0.1070859283208847, 0.09959358721971512] \n",
      "epoch[16] iter[140] loss: [0.1261908859014511, 0.23302394151687622] \n",
      "epoch[16] iter[160] loss: [0.10968554019927979, 0.15521426498889923] \n",
      "epoch[16] iter[180] loss: [0.10019786655902863, 0.14191864430904388] \n",
      "epoch[16] iter[200] loss: [0.09931234270334244, 0.12103894352912903] \n",
      "epoch[16] iter[220] loss: [0.12628065049648285, 0.12701714038848877] \n",
      "epoch[16] iter[240] loss: [0.10560106486082077, 0.09334856271743774] \n",
      "epoch[16] iter[260] loss: [0.12549763917922974, 0.16706855595111847] \n",
      "epoch[16] iter[280] loss: [0.09338606894016266, 0.08775746077299118] \n",
      "epoch[16] iter[300] loss: [0.09548183530569077, 0.09385275095701218] \n",
      "epoch[16] iter[320] loss: [0.09880585968494415, 0.08216163516044617] \n",
      "epoch[16] iter[340] loss: [0.09332744777202606, 0.08380743116140366] \n",
      "epoch[16] iter[360] loss: [0.10970296710729599, 0.11753657460212708] \n",
      "epoch[16] iter[380] loss: [0.10161371529102325, 0.09181435406208038] \n",
      "Finish epoch[16], time elapsed[282.5854434967041]\n",
      "epoch[17] iter[0] loss: [0.09498118609189987, 0.0995904803276062] \n",
      "epoch[17] iter[20] loss: [0.12087960541248322, 0.10680346935987473] \n",
      "epoch[17] iter[40] loss: [0.09008008241653442, 0.15659183263778687] \n",
      "epoch[17] iter[60] loss: [0.1107291728258133, 0.14951345324516296] \n",
      "epoch[17] iter[80] loss: [0.09654553979635239, 0.09919091314077377] \n",
      "epoch[17] iter[100] loss: [0.10696952044963837, 0.12225425988435745] \n",
      "epoch[17] iter[120] loss: [0.11487870663404465, 0.08905361592769623] \n",
      "epoch[17] iter[140] loss: [0.10694490373134613, 0.15388703346252441] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[17] iter[160] loss: [0.10791702568531036, 0.10078510642051697] \n",
      "epoch[17] iter[180] loss: [0.1112760379910469, 0.10839395970106125] \n",
      "epoch[17] iter[200] loss: [0.10062745958566666, 0.09941805899143219] \n",
      "epoch[17] iter[220] loss: [0.10446739941835403, 0.09802116453647614] \n",
      "epoch[17] iter[240] loss: [0.11910661309957504, 0.13601484894752502] \n",
      "epoch[17] iter[260] loss: [0.09137111157178879, 0.0747222900390625] \n",
      "epoch[17] iter[280] loss: [0.10172589123249054, 0.09916910529136658] \n",
      "epoch[17] iter[300] loss: [0.1136292889714241, 0.12790223956108093] \n",
      "epoch[17] iter[320] loss: [0.12115475535392761, 0.11727125942707062] \n",
      "epoch[17] iter[340] loss: [0.0872260332107544, 0.09709151089191437] \n",
      "epoch[17] iter[360] loss: [0.09927301853895187, 0.08252526819705963] \n",
      "epoch[17] iter[380] loss: [0.09995254129171371, 0.11274127662181854] \n",
      "Finish epoch[17], time elapsed[281.4574875831604]\n",
      "epoch[18] iter[0] loss: [0.08882640302181244, 0.08465209603309631] \n",
      "epoch[18] iter[20] loss: [0.10309042781591415, 0.08551579713821411] \n",
      "epoch[18] iter[40] loss: [0.10474062711000443, 0.08395504951477051] \n",
      "epoch[18] iter[60] loss: [0.097758948802948, 0.16003337502479553] \n",
      "epoch[18] iter[80] loss: [0.10116799175739288, 0.07827379554510117] \n",
      "epoch[18] iter[100] loss: [0.09970179945230484, 0.08028551191091537] \n",
      "epoch[18] iter[120] loss: [0.08712416142225266, 0.07703419029712677] \n",
      "epoch[18] iter[140] loss: [0.10595180094242096, 0.13096672296524048] \n",
      "epoch[18] iter[160] loss: [0.09105319529771805, 0.13385429978370667] \n",
      "epoch[18] iter[180] loss: [0.09895763546228409, 0.08417490869760513] \n",
      "epoch[18] iter[200] loss: [0.12414096295833588, 0.11353371292352676] \n",
      "epoch[18] iter[220] loss: [0.09581726789474487, 0.080407053232193] \n",
      "epoch[18] iter[240] loss: [0.08773990720510483, 0.11221921443939209] \n",
      "epoch[18] iter[260] loss: [0.10031596571207047, 0.08173076063394547] \n",
      "epoch[18] iter[280] loss: [0.10605695843696594, 0.0926760733127594] \n",
      "epoch[18] iter[300] loss: [0.09483601152896881, 0.07912840694189072] \n",
      "epoch[18] iter[320] loss: [0.10638214647769928, 0.10193348675966263] \n",
      "epoch[18] iter[340] loss: [0.09129991382360458, 0.08059875667095184] \n",
      "epoch[18] iter[360] loss: [0.09456826001405716, 0.09669994562864304] \n",
      "epoch[18] iter[380] loss: [0.09580698609352112, 0.0966876894235611] \n",
      "Finish epoch[18], time elapsed[280.58758425712585]\n",
      "epoch[19] iter[0] loss: [0.10078036785125732, 0.08337792009115219] \n",
      "epoch[19] iter[20] loss: [0.10462358593940735, 0.09075251966714859] \n",
      "epoch[19] iter[40] loss: [0.10399601608514786, 0.13164694607257843] \n",
      "epoch[19] iter[60] loss: [0.10964729636907578, 0.1324114054441452] \n",
      "epoch[19] iter[80] loss: [0.11023565381765366, 0.09717318415641785] \n",
      "epoch[19] iter[100] loss: [0.11055425554513931, 0.10804936289787292] \n",
      "epoch[19] iter[120] loss: [0.09248106926679611, 0.09239599853754044] \n",
      "epoch[19] iter[140] loss: [0.10985362529754639, 0.1550479680299759] \n",
      "epoch[19] iter[160] loss: [0.08741297572851181, 0.07025916874408722] \n",
      "epoch[19] iter[180] loss: [0.09798549860715866, 0.1088213175535202] \n",
      "epoch[19] iter[200] loss: [0.10594278573989868, 0.08981737494468689] \n",
      "epoch[19] iter[220] loss: [0.0967501550912857, 0.08610708266496658] \n",
      "epoch[19] iter[240] loss: [0.10572163760662079, 0.11825980246067047] \n",
      "epoch[19] iter[260] loss: [0.10461384803056717, 0.16181841492652893] \n",
      "epoch[19] iter[280] loss: [0.0986495092511177, 0.12116933614015579] \n",
      "epoch[19] iter[300] loss: [0.09866528958082199, 0.09515578299760818] \n",
      "epoch[19] iter[320] loss: [0.10041021555662155, 0.09346070140600204] \n",
      "epoch[19] iter[340] loss: [0.11498124152421951, 0.11603590846061707] \n",
      "epoch[19] iter[360] loss: [0.10281986743211746, 0.09766219556331635] \n",
      "epoch[19] iter[380] loss: [0.09843640774488449, 0.08227340877056122] \n",
      "Finish epoch[19], time elapsed[281.5515847206116]\n"
     ]
    }
   ],
   "source": [
    "loss_all = []\n",
    "for epoch in range(num_epochs):\n",
    "    LaneNet_model.train()\n",
    "    ts = time.time()\n",
    "    for iter, batch in enumerate(data_loader_train):\n",
    "        input_image = Variable(batch[0]).to(device)\n",
    "        binary_labels = Variable(batch[1]).to(device)\n",
    "        instance_labels = Variable(batch[2]).to(device)\n",
    "        \n",
    "        binary_final_logits, instance_embedding = LaneNet_model(input_image)\n",
    "        # loss = LaneNet_model.compute_loss(binary_logits=binary_final_logits, binary_labels=binary_labels,\n",
    "        #                               instance_logits=instance_embedding, instance_labels=instance_labels, delta_v=0.5, delta_d=3)\n",
    "        binary_segmenatation_loss, instance_segmenatation_loss = criterion(binary_logits=binary_final_logits, binary_labels=binary_labels,\n",
    "                                       instance_logits=instance_embedding, instance_labels=instance_labels, delta_v=0.5, delta_d=3)\n",
    "        \n",
    "        # binary_segmenatation_loss = criterion(binary_final_logits, binary_labels)\n",
    "        loss = 1*binary_segmenatation_loss + 1*instance_segmenatation_loss\n",
    "        optimizer.zero_grad()\n",
    "        loss_all.append(loss.item())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if iter % 20 == 0:\n",
    "            print(\"epoch[{}] iter[{}] loss: [{}, {}] \".format(epoch, iter, binary_segmenatation_loss.item(), instance_segmenatation_loss.item()))\n",
    "    lr_scheduler.step()\n",
    "    print(\"Finish epoch[{}], time elapsed[{}]\".format(epoch, time.time() - ts))\n",
    "    torch.save(LaneNet_model.state_dict(), \n",
    "                       f\"TUSIMPLE/Lanenet_output/lanenet_epoch_{epoch}_batch_{8}.model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show the Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fba09211610>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAdyUlEQVR4nO3deXxU9b3/8dcnCQmELSxBWQ0o4lIVMa5Ypa5YrO39tfdWW/uzra0/ra1a76+9+Ku12tqqrdeqrbVSrbVXRFu1qKAggihuYFhkXyJLCFsSCAGyTma+vz/mJAZCllkyM2d4Px+PPDJz5sw5bzLDe86c+c455pxDRETSU0ayA4iISNdRyYuIpDGVvIhIGlPJi4ikMZW8iEgay0rkygYOHOgKCgoSuUoREd9bvHhxhXMuP5r7JrTkCwoKKCoqSuQqRUR8z8y2RHtf7a4REUljKnkRkTSmkhcRSWMqeRGRNKaSFxFJYyp5EZE0ppIXEUljvij59bv28/HmPcmOISLiOwn9MlS0Lvv9uwBsvn9SkpOIiPiLL7bkRUQkOip5EZE0ppIXEUljKnkRkTSmkhcRSWMqeRGRNOaLkh/UOyfZEUREfKnDkjezv5pZmZmtbDGtv5nNMbMN3u9+XRny7FEDGDmwZ1euQkQkLXVmS/5vwMRDpk0G5jrnRgNzvetdxrpy4SIiaazDknfOvQscekyBLwPPeJefAb4S51yHy9HVqxARSTvR7pM/yjm3A8D7PSh+kVozA1W8iEjkuvzYNWZ2A3ADwIgRI6Jaxrvry6msCcQzlojIESHaLfldZjYYwPtd1taMzrkpzrlC51xhfn5+VCtrKnjtshERiUy0Jf8qcJ13+TrglfjEaV9jSCUvIhKJzgyhnAZ8CIwxs1Izux64H7jUzDYAl3rXu1wgGErEakRE0kaH++Sdc9e0cdPFcc7SoUBQW/IiIpHwxTdef3TRcQA0akteRCQivij5IXk9AG3Ji4hEyhcln5UR/s6r9smLiETGFyXfLTMcUyUvIhIZX5W8hlCKiETGFyWflandNSIi0fBFyXdrLnltyYuIRMIXJV8fCG/Bl1bWJDmJiIi/+KLkZ67YAcBdr6xKchIREX/xRclf8bnBAOypbkhyEhERf/FFyR/dV+d4FRGJhi9KvmkIpYiIRMYX7ZmZobO8iohEwxcln2EqeRGRaKjkRUTSmE9KPtkJRET8yRcln99bo2tERKLR4ZmhUkFebjZm8O3zCpIdRUTEV3yxJQ/QK8cXr0ciIinFNyWfYYbT8clERCLio5KHkFpeRCQivin5ypoA89aWJTuGiIiv+KbkAUora5MdQUTEV3xV8iIiEhmVvIhIGlPJi4ikMZW8iEga813JNwZDyY4gIuIbviv5oMbKi4h0Wkwlb2Y/NrNVZrbSzKaZWfd4BRMRkdhFXfJmNhS4BSh0zn0OyASujlcwERGJXay7a7KAHmaWBeQC22OPJCIi8RJ1yTvntgEPAiXADqDKOffmofOZ2Q1mVmRmReXl5dEnFRGRiMWyu6Yf8GVgJDAE6Glm1x46n3NuinOu0DlXmJ+fH31SERGJWCy7ay4BNjnnyp1zAeBl4Lz4xBIRkXiIpeRLgHPMLNfMDLgYWBOfWCIiEg+x7JNfCLwILAFWeMuaEqdcIiISBzGdU8859wvgF3HK0imV1QGO7puZyFWKiPiW777xur1Kx5QXEeks35W8jmogItJ5vit5UMuLiHSW70peW/IiIp3nv5JPdgARER/xXclvrqhOdgQREd/wXcnPW1uW7AgiIr7hu5IPBLXDRkSks3xX8sGQTv8nItJZviv58GFyRESkM3xX8iIi0nm+K3mngfIiIp3mu5JvDKnkRUQ6y3clv2BDRbIjiIj4hm9K/vZLj092BBER3/FNyfft0S3ZEUREfMc3JS8iIpHzTclreLyISOR8U/IiIhI5lbyISBpTyYuIpDGVvIhIGlPJi4ikMZW8iEga803JawSliEjkfFPyIiISOd+UvI4+KSISOd+UfFAlLyISMZW8iEgai6nkzSzPzF40s7VmtsbMzo1XsEPlZmd21aJFRNJWrFvyjwCznHMnAKcBa2KPdHhfOm1IVy1aRCRtZUV7RzPrA1wAfBvAOdcANMQnVmu52VFHFRE5YsWyJT8KKAeeNrOlZvakmfU8dCYzu8HMisysqLy8POqVZWf55uMDEZGUEUtzZgHjgMedc6cD1cDkQ2dyzk1xzhU65wrz8/NjWJ2IiEQqlpIvBUqdcwu96y8SLn0REUkRUZe8c24nsNXMxniTLgZWxyWViIjERayfZv4ImGpm2cBG4DuxRxIRkXiJqeSdc8uAwjhlERGRONOQFRGRNKaSFxFJYyp5EZE0ppIXEUljKnkRkTSmkhcRSWMqeRGRNOarkv/quGEMzeuR7BgiIr7hq5LPMHBOZ4gSEeksn5W8obMAioh0nr9KPgNC2pIXEek0X5X82p37Kdtfn+wYIiK+4auSX1qyF9B+eRGRzvJVyTfZtU9b8yIineHLki/asifZEUREfMGXJV9aWZvsCCIivuDLkv/vN9clO4KIiC/4suQzzJIdQUTEF3xZ8pkZKnkRkc7wZ8lrS15EpFN8WfK5OZnJjiAi4gu+KvmvjB0CQHaWr2KLiCSNr9oyw9sXHwolOYiIiE/4q+S1L15EJCK+KvmmiteRKEVEOsdXJd+0Ja+SFxHpHH+VvJdWJw4REekcX5V805egynVMeRGRTvFVyR/Tv2eyI4iI+ErMJW9mmWa21MxmxCNQeyadOrirVyEiklbisSV/K7AmDsvpkI5ZIyISmZhK3syGAZOAJ+MTp30aJy8iEplYt+QfBn4KtPkdVDO7wcyKzKyovLw8ppVpS15EJDJRl7yZXQmUOecWtzefc26Kc67QOVeYn58f7eoAUMeLiEQmli358cBVZrYZeB64yMyejUuqNhhqeRGRSERd8s65O5xzw5xzBcDVwDzn3LVxS3Y46ngRkYj4apx875ysZEcQEfGVuLSmc24+MD8ey2pPhnbKi4hExFdb8iIiEhmVvIhIGlPJi4ikMd+WfEjHGxYR6ZBvS/7FJaXJjiAikvJ8W/KvfbI92RFERFKeb0t+wYaKZEcQEUl5vi15ERHpmEpeRCSNqeRFRNKYSl5EJI35uuTrG4PJjiAiktJ8XfJO34cSEWmXSl5EJI35uuRDankRkXap5EVE0pjvSv7Mgn7Nl0OhJAYREfEB35X8s987u/mytuRFRNrnu5LPycpsvhxUyYuItMt3Jd+StuRFRNrn65JHHS8i0i5fl3xx2YFkRxARSWm+LvlvPLkw2RFERFKar0teRETap5IXEUljvi/5zRXVrN6+L9kxRERSUlayA8RqwoPzAdh8/6TkBhERSUG+35IXEZG2pU3JX/LQOxRMnklVbSDZUUREUkbUJW9mw83sbTNbY2arzOzWeAZrz4P/flqraU1j5rfuqUlUDBGRlBfLlnwj8J/OuROBc4Cbzeyk+MRq31fHDU3EakREfC/qknfO7XDOLfEu7wfWAAlpXzNrJ1ciEoiI+ENc9smbWQFwOtDqK6hmdoOZFZlZUXl5eTxW167agE7uLSLSJOaSN7NewEvAbc65VgPWnXNTnHOFzrnC/Pz8WFfXoe//vajL1yEi4hcxlbyZdSNc8FOdcy/HJ1JsNLpGROQzsYyuMeApYI1z7qH4RYrdA7PWsn7X/mTHEBFJuli25McD3wIuMrNl3s8X45SrQzdeeGybtz0+/1P+44kPExVFRCRlRX1YA+fce0Dbw1y62E0TjuXP73za5u31AZ3lW0TEt994bWcUJaBRNiIi4OOS75nt+2OriYh0Od+WfGaGse7eicmOISKS0nxb8gA5WZnt3n7XKytZt1OjbETkyOXrku/I3z/cwo9fWJbsGCIiSeP7kh89qFe7t6/esY/H3i5OUBoRkdTi+5Kf+r2zO5znqfc2JSCJiEjq8X3JD+rTvcN56jScUiQlOOeYsXw7wZAOF5sovi95gGevb39rvqZBJS+SCqYv28YPn1vKX/XuOmHSouTPHz2Qgb1ykh1DRDpQsb8BgF376pKc5MiRFiUPkJXR/ldgtctGRI5EaVPyj187rt3bT/j5LEora/jb+5sYf/+8BKUSEUmutDk2wOkj+vHW7RdwyUPvtjnP+Q+83Xw5EAzRLTNtXuNERA4rrVruuEG9ueq0IZ2ad87qXV2cRkQO5dComkRLq5IHePjrY9nw6ys6nO8HU5ewdU9Nm7dPfmk5P5q2NJ7RREQSLu1KPiPDOr0b5qONu3lw9jqca7118fzHW3ntk+0cqG+Md0SRI5Yl7xQUR6y0K/lI/OTF5fzx7WK27a1tnnbf62somDyz+fpvXl8DQLVX9s456hs1UkdE/CFtS/6Rq8fy52vHseyuSzuc9/wH3ub3c9YD8MS7Gw+67bmFJTy5YCMn/2I2K7dV8dvZ6xhz5yzqAkG2763luYUlXZI/UXYfqOfuV1cRCOpMWiLpKG1G1xzqy2OHRjT/I3M38MjcDYe97d6Z4a35x9/5lJnLdwDwnac/5sONuwGY+Lmj6d8zu81l76yq4/Z/LOPxb55B39xuEeXqar+csZpXlm2nsKAfV57auQ+tRcQ/0nZLvqVRA3vGZTlNBQ80FzxAQ2OI7Xtruee1VTQ0hreI99Y08I+Pt/Lkgo2cc99cPvh0N/9aWnrY5TrnKJg8kz/Nb320zJXbqiiYPJMn2jmfbSwavWOIHOZjCUkDe6ob+GTr3mTHkCQ6Ikr+79efxdjheV22/L8s2Mh598/j6fc3c/ydb/DsR1u45fll/PSl5c3vAlraVFHN+Q/Mo7QyPLqn6WBND85e1zzPjqpalpRUcuUf3gPgvjfWNt/22NvFnHTXrObrLy8pZUVpVXThj5ByX7hxNyu3Rfk38rF/+9P7fPmx95MdQ5LoiCj5Yf1ymX7zeH73tVO7ZPmHHsr4zukreXd9eav5nvlwC3trGvjCg/Mprazl9hc+AWDWqp0AhFy4wEMhx7n3zeN//emDg+7f0BhixvLt/G72Omoagjz2djHvrC/n9n98wpf+GH4xKNldwwefVlAweeZBHyBD+B3D4UYSQesTo/9+znre21DRqX9/KOQo2rznoPU8OncDi7fsYdKjCyjb3/ZxSiqrGxJyRMKvT/mo+QUzmTZXVLOx/EDC1rdld9vDhDvr3fXlXDPlI0I6cqQvpe0++cP598LhTFtUwpKS5Lx93VRRzdhfzmm+vmjznlZF/LvZ63hl2bbD3v/4O99oNW9L985YzZOHvOBs2LUfM2NzRTXf+3sR3zx7BPdcdTJ1jSHK9tUxc0V4F9QPn1vKtEUlXHXaEL5+5ojmzyc23z+peVn/KNrKxScMYoB3MLgPiis4o6Af0xaWcPdrq/nzteO48dklHNUnh1376nnI+6ee9eu5PHv92Zwzqj9ZLYa3VtUGOP1Xc/j+50fyk8tPYPWOfZw8pA+vr9jB2SMHcHTfjg8j3ZZd++p4bmEJt10ymtooj1v0yrJtDOrdnXOPHdDqtrpAkIWb9nDh8fk0NIbIzDAy2zh+0sbyAywvreIrpw9lwoPzgYP/rqnu5ueWsL+ukf11jXH7TKm0spaqmkDcllcXCJIZwfDpSC3cuJsTh/ShT/fU+kytM46okgd4+QfjeWPFDm6auiTZUdq0fld0W3qHFjzApb8/+DAPUxeWMNUbEXT5yUcddNv7xbt5v3g3//XSiuZpv565mr8s2ER2ZgYN3gicwmP6UbSlstW6bnw2/Dfdta++1W3XPrWQWy4eze2XHk8w5Ji6cAvjjxsIwIuLS3nyvU04B5ecOIi31pQB8LfvnMn1zxQRDDlm/Oh8Pje070HL/Pn0lVTVBnjgq6fyz8VbufbsY8jwivbHLyzjg093M2FMPnO95bW0duc+QiE4aUifVrc1ufX58Kkji399xUEvTgD3vLaaaYtKeP2Wz/PFRxdw+clH8cS3Cg+7nCseWUB9Y4ivnB7ZYIC2OOfYUVXHkLwecVleS8VlB9hfF+D0Ef0Oe3swFH43aNb2i1qTpSWVOGDcYZY1a9VOirbsoejOjke/dcYJP5/FmQX9+OeN58VleS0dqG/k61M+4txRA5h2wzlxX35XO+JKHuCKUwY3X77y1MHMaPGB6pFk9qqOD+3wlwXhF46GFkMsD1fwnfHo3A3MWb2LSacczYNvrm+eXlkTaL78VotC/vbTHzdfvvIP73HVaUOoaQjypdMGU9sQ5H8+2gLAq59sB+CuV1Zx1sj+LNr02a6jW59fRkmLbza/vKSUCWMGMfHhBQA8/Z0zmbtmFzv21rFtby15ud0Y3LcH/1r62bup4372BkPzerBtby03TTiWiScfzbRF4RfKLz4aXs7sVbsomDyT6849hhMG9+Gas0Z403dS730Y/6a3Ww5g654ahvfPpTEY4vWVOwmFHGeP6s/gvj34w9wN/Pec9ay7dyK/fG01E8YM4i/vbqQ+GCK/V3bz32jWbZ/n1zPX8N3zR/KFMYNYWlLJsH655Pdufdjtkt01DOiVTc+c1v/laxoa+eO8Ym69ZDSXPPQOEH6n8fT7m1hRWsX+uvB3RE7/1Zs07bHJycpg3b3hb5ZvqqimuOwAd72ykof+YyzjjskjOzODf/N2N0751hnsqKprtVuu4kBD8+V7XlvFup37ee77B5foS4tLmb5sG09ddyYffFrBhDGDWuVv8vHmg5+Xf3t/E7k5WVx8wiB6dc8iww6/pV8XCHLb88v42aQTGd4/96DbCibP5EveoVLW7NzX5roP50B9I7+fs56fXD6G7t0yI7pvPFlb+2i7QmFhoSsqKkrY+tqzYdd+irZUcs1ZI5i+dBvb9tZy3rEDmp+YIrHKzspoHm3VlgyDRO7qPmZALv818QRe+Hgr76wvZ/ndl/Hz6St5Zdn2g+Zb9P8u5qzfzG13WY99Yxzrdu7j0XnxO4fy1WcO56ITBjF/fTlFm/e0elfbu3sWJw3uw0LvhXzz/ZPYWVXHOfeFs6791UT+ubiUn09fedjlr7t3IjlZmTjnaAiGqAuEeHlJKfe8thqAr50xjBcXh0fBrbj7Mk65+82D7j/7tgsYc3Tv5us1DY3kZGVSGwgye+VO9lQ3UFx2gPOOG8Dkl1ZQGwiSk5XBSUP68NR1Z7Y71Lo9ZrbYOXf4t4od3fdILfmO7KiqZcH6CkYMyKVbpnHykL584y8fsaRkLz+5fAxfO2MYUxeW8GgbY+tFRFqa+58Xcmx+r6juq5JPomDIsa82wPJtVVwweiDmDVPZW9NAn+7dCIRC/GbmGq4aO5S6QJCH31rPmQX9+dP88Lj3Wy46jhsnHMury7Yz+eUV7a2KYwbk8v3Pj2LJlkpe9nYnPPPds7jur4u69h8pIjGbfvP4qIdyJ63kzWwi8AiQCTzpnLu/vfnTseTjKRRyHGhojOoTfOccry3fwfhjB9C7ezcCwVDz/tctu6vZVlnLed4Hndv31tIzJ4vc7EzeK67g5CF9yM3O4r0NFUwYk48ZVFYH6N4tg5ysTP7vi5+w1Nu1tamimitOGcxxg3pRXd9IIBjitGF5LN9WxdjhedQ3BllaspdjBuTy6Nxipi0qYezwPG688Fh2V9fz1updbN5dw6aKau6cdCIAizbtYeyIPH476+DRQs/fcA7z1pZRVRPghaKtfPPsEdQFQry0JPx2+s5JJ/K5oX15fcUOAsEQ0xZtbfdv9IUx+by9rvXQVkm8S048irfWHFmH+15216Xk5fpod42ZZQLrgUuBUuBj4Brn3Oq27qOSl3QUCrnmUT1drbYhSFamUVnTwKDe3SnbX0ePbpn09jYMAsEQq7bv49ShfdvM1BgMkZWZwfSl29hQtp/xxw7k1OF5zdN7eRsHzjlCDiprGqhtCBIMOQq8b49XVjdQEwiSYRBodPTIzqR8fz2j8ntSHwiR0y08Gis7M6PNDx0bGkNs2V3NcYN6YWZU1zfyp/nF3Hrx8ZiFv1MwcmBPsjIzqKoJEHKOfj2zqaoJMGvVDrplZnDa8DxG9M9l0aY99MjOpKomwAmDe9MrJwsH9OnejbJ9dQSdY93O/eRmZ7GvNsBNUxczamAvxh83kP9z4SgyzHDOMWvVTk4Z2pepC0u45aLRlFbWUFjQn0/LD9DQGCIQDDEqvxePvLWekQN7csqwPP4wbwO79tWTm53JSYP7cNslo9lRVUdNQ5CeOeFpgaAjOyv64Z3JKvlzgbudc5d71+8AcM7d19Z9VPIiIpGLpeRj+ebAUKDl++NSb5qIiKSIWEr+cO8FW70tMLMbzKzIzIrKy7U/VEQkkWIp+VJgeIvrw4Dth87knJvinCt0zhXm5+fHsDoREYlULCX/MTDazEaaWTZwNfBqfGKJiEg8RH1YA+dco5n9EJhNeAjlX51zq+KWTEREYhbTsWucc68Dr8cpi4iIxNkRcTx5EZEjlUpeRCSNJfTYNWZWDmyJ8u4Dgc6dqijxUjkbpHa+VM4GqZ0vlbNBaufzW7ZjnHNRDU9MaMnHwsyKov3GV1dL5WyQ2vlSORukdr5Uzgapne9IyqbdNSIiaUwlLyKSxvxU8lOSHaAdqZwNUjtfKmeD1M6XytkgtfMdMdl8s09eREQi56cteRERiZBKXkQkjfmi5M1sopmtM7NiM5ucoHX+1czKzGxli2n9zWyOmW3wfvfzppuZPerlW25m41rc5zpv/g1mdl2csg03s7fNbI2ZrTKzW1MsX3czW2Rmn3j57vGmjzSzhd66XvAObIeZ5XjXi73bC1os6w5v+jozuzwe+bzlZprZUjObkUrZzGyzma0ws2VmVuRNS4nH1Vtunpm9aGZrveffuamQz8zGeH+zpp99ZnZbKmRrsdwfe/8fVprZNO//Sdc/75xzKf1D+OBnnwKjgGzgE+CkBKz3AmAcsLLFtN8Ck73Lk4EHvMtfBN4gfIz9c4CF3vT+wEbvdz/vcr84ZBsMjPMu9yZ8GsaTUiifAb28y92Ahd56/wFc7U3/M3CTd/kHwJ+9y1cDL3iXT/Ie7xxgpPc8yIzT43s78Bwww7ueEtmAzcDAQ6alxOPqLfsZ4Hve5WwgL5XyecvPBHYCx6RKNsInVNoE9GjxfPt2Ip53cfmjduUPcC4wu8X1O4A7ErTuAg4u+XXAYO/yYGCdd/kJwue3PWg+4BrgiRbTD5ovjjlfIXyu3ZTLB+QCS4CzCX+LL+vQx5XwkUzP9S5nefPZoY91y/lizDQMmAtcBMzw1pUq2TbTuuRT4nEF+hAuKkvFfC2Wdxnwfipl47Mz6fX3nkczgMsT8bzzw+6aVDrN4FHOuR0A3u9B3vS2MnZ5du9t3OmEt5ZTJp+3O2QZUAbMIbzFsdc513iYdTXn8G6vAgZ0Yb6HgZ8CIe/6gBTK5oA3zWyxmd3gTUuVx3UUUA487e3qetLMeqZQviZXA9O8yymRzTm3DXgQKAF2EH4eLSYBzzs/lHynTjOYZG1l7NLsZtYLeAm4zTm3r71Z28jRZfmcc0Hn3FjCW81nASe2s66E5TOzK4Ey59zilpNTIZtnvHNuHHAFcLOZXdDOvInOlkV4F+bjzrnTgWrCu0DakvDnnbdP+yrgnx3N2kaGLsnmfRbwZcK7WIYAPQk/xm2tK275/FDynTrNYILsMrPBAN7vMm96Wxm7LLuZdSNc8FOdcy+nWr4mzrm9wHzC+z3zzKzpHAYt19Wcw7u9L7Cni/KNB64ys83A84R32TycItlwzm33fpcB/yL8Apkqj2spUOqcW+hdf5Fw6adKPggX5xLn3C7veqpkuwTY5Jwrd84FgJeB80jA884PJZ9Kpxl8FWj6tP06wvvCm6b/b+8T+3OAKu+t4WzgMjPr572SX+ZNi4mZGfAUsMY591AK5ss3szzvcg/CT/A1wNvA19rI15T7a8A8F97h+CpwtTfSYCQwGlgUSzbn3B3OuWHOuQLCz6V5zrlvpkI2M+tpZr2bLhN+PFaSIo+rc24nsNXMxniTLgZWp0o+zzV8tqumKUMqZCsBzjGzXO//b9Pfruufd/H6sKMrfwh/Er6e8H7dnyVondMI7zsLEH71vJ7wPrG5wAbvd39vXgMe8/KtAApbLOe7QLH38504ZTuf8Fu05cAy7+eLKZTvVGCpl28lcJc3fZT3hCwm/HY6x5ve3bte7N0+qsWyfublXgdcEefHeAKfja5JejYvwyfez6qm53qqPK7ecscCRd5jO53wCJSUyEf4Q/7dQN8W01Iim7fce4C13v+J/yE8QqbLn3c6rIGISBrzw+4aERGJkkpeRCSNqeRFRNKYSl5EJI2p5EVE0phKXkQkjankRUTS2P8HZ4DZThIky1kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pylab as plt\n",
    "plt.plot(loss_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
